{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "# Load environment variables from openai.env file\n",
    "load_dotenv(\"openai.env\")\n",
    "\n",
    "# Read the OPENAI_API_KEY from the environment\n",
    "api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "api_base = os.getenv(\"OPENAI_API_BASE\")\n",
    "os.environ[\"OPENAI_API_KEY\"] = api_key\n",
    "os.environ[\"OPENAI_API_BASE\"] = api_base"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第一个Agent\n",
    "- 注意当前使用的langchain版本为0.1.4，调用方式和之前旧版本有所差异"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m你好！有什么我可以帮助你的吗？\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': '你好', 'output': '你好！有什么我可以帮助你的吗？'}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from langchain_openai import ChatOpenAI\n",
    "from langchain import hub\n",
    "from langchain.agents import load_tools\n",
    "from langchain.agents import create_openai_functions_agent #不同的agent有不同的创建方式\n",
    "from langchain.agents import AgentExecutor\n",
    "#创建LLM \n",
    "llm = ChatOpenAI(model_name=\"gpt-4\", temperature=0)\n",
    "#定义agent的prompt\n",
    "#https://smith.langchain.com/hub/hwchase17/openai-functions-agent\n",
    "prompt = hub.pull(\"hwchase17/openai-functions-agent\")\n",
    "#定义工具,加载预制的工具,注意有的工具需要提供LLM\n",
    "tools = load_tools([\"llm-math\"], llm=llm)\n",
    "#创建agent\n",
    "agent = create_openai_functions_agent(llm, tools, prompt)\n",
    "#定义agent的执行器，这里注意与老版本的不同\n",
    "agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)\n",
    "agent_executor.invoke({\"input\": \"你好\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agent\n",
    "- 中间步骤处理\n",
    "- 提示词\n",
    "- 模型配置(停止符必要的话)\n",
    "- 输出解析器\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain import hub\n",
    "from langchain.agents import AgentExecutor, tool\n",
    "from langchain.agents.output_parsers import XMLAgentOutputParser\n",
    "from langchain_openai import ChatOpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[StructuredTool(name='search', description='search(query: str) -> str - 当需要了解最新的天气信息的时候才会使用这个工具。', args_schema=<class 'pydantic.v1.main.searchSchemaSchema'>, func=<function search at 0x15a7b9800>)]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#配置模型\n",
    "model = ChatOpenAI(\n",
    "    model=\"gpt-4-1106-preview\",\n",
    "    temperature=0,\n",
    "    )\n",
    "#可用工具\n",
    "@tool\n",
    "def search(query: str) -> str:\n",
    "    \"\"\"当需要了解最新的天气信息的时候才会使用这个工具。\"\"\"\n",
    "    return \"晴朗,32摄氏度,无风\"\n",
    "tool_list = [search]\n",
    "tool_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ChatPromptTemplate(input_variables=['agent_scratchpad', 'input', 'tools'], partial_variables={'chat_history': ''}, messages=[HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['agent_scratchpad', 'chat_history', 'input', 'tools'], template=\"You are a helpful assistant. Help the user answer any questions.\\n\\nYou have access to the following tools:\\n\\n{tools}\\n\\nIn order to use a tool, you can use <tool></tool> and <tool_input></tool_input> tags. You will then get back a response in the form <observation></observation>\\nFor example, if you have a tool called 'search' that could run a google search, in order to search for the weather in SF you would respond:\\n\\n<tool>search</tool><tool_input>weather in SF</tool_input>\\n<observation>64 degrees</observation>\\n\\nWhen you are done, respond with a final answer between <final_answer></final_answer>. For example:\\n\\n<final_answer>The weather in SF is 64 degrees</final_answer>\\n\\nBegin!\\n\\nPrevious Conversation:\\n{chat_history}\\n\\nQuestion: {input}\\n{agent_scratchpad}\"))])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#提示词模版\n",
    "# https://smith.langchain.com/hub\n",
    "# Get the prompt to use - you can modify this!\n",
    "prompt = hub.pull(\"hwchase17/xml-agent-convo\")\n",
    "prompt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "中间步骤，实现一个log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_intermediate_steps(intermediate_steps):\n",
    "    log = \"\"\n",
    "    for action, observation in intermediate_steps:\n",
    "        log += (\n",
    "            f\"<tool>{action.tool}</tool><tool_input>{action.tool_input}\"\n",
    "            f\"</tool_input><observation>{observation}</observation>\"\n",
    "        )\n",
    "    return log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将工具列表插入到模版中\n",
    "def convert_tools(tools):\n",
    "    return \"\\n\".join([f\"{tool.name}: {tool.description}\" for tool in tools])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = (\n",
    "    {\n",
    "        \"input\": lambda x: x[\"input\"],\n",
    "        \"agent_scratchpad\": lambda x: convert_intermediate_steps(\n",
    "            x[\"intermediate_steps\"]\n",
    "        ),\n",
    "    }\n",
    "    | prompt.partial(tools=convert_tools(tool_list))\n",
    "    | model.bind(stop=[\"</tool_input>\", \"</final_answer>\"])\n",
    "    | XMLAgentOutputParser()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RunnableAssign(mapper={\n",
       "  agent_scratchpad: RunnableLambda(lambda x: format_to_openai_function_messages(x['intermediate_steps']))\n",
       "})\n",
       "| ChatPromptTemplate(input_variables=['agent_scratchpad', 'input'], input_types={'chat_history': typing.List[typing.Union[langchain_core.messages.ai.AIMessage, langchain_core.messages.human.HumanMessage, langchain_core.messages.chat.ChatMessage, langchain_core.messages.system.SystemMessage, langchain_core.messages.function.FunctionMessage, langchain_core.messages.tool.ToolMessage]], 'agent_scratchpad': typing.List[typing.Union[langchain_core.messages.ai.AIMessage, langchain_core.messages.human.HumanMessage, langchain_core.messages.chat.ChatMessage, langchain_core.messages.system.SystemMessage, langchain_core.messages.function.FunctionMessage, langchain_core.messages.tool.ToolMessage]]}, messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template='You are a helpful assistant')), MessagesPlaceholder(variable_name='chat_history', optional=True), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['input'], template='{input}')), MessagesPlaceholder(variable_name='agent_scratchpad')])\n",
       "| RunnableBinding(bound=ChatOpenAI(client=<openai.resources.chat.completions.Completions object at 0x14e8aef10>, async_client=<openai.resources.chat.completions.AsyncCompletions object at 0x15a499fd0>, model_name='gpt-4', temperature=0.0, openai_api_key='sk-pQ7osw5XrDY1Ejsx99EcEe5f58D04dDeB387B8E6B3C10bDb', openai_api_base='https://ai-yyds.com/v1', openai_proxy='https://ai-yyds.com/v1'), kwargs={'functions': [{'name': 'Calculator', 'description': 'Useful for when you need to answer questions about math.', 'parameters': {'properties': {'__arg1': {'title': '__arg1', 'type': 'string'}}, 'required': ['__arg1'], 'type': 'object'}}]})\n",
       "| OpenAIFunctionsAgentOutputParser()"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#执行agent\n",
    "agent_executor = AgentExecutor(agent=agent, tools=tool_list, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m<tool>search</tool><tool_input>北京今天的天气\u001b[0m\u001b[36;1m\u001b[1;3m晴朗,32摄氏度,无风\u001b[0m\u001b[32;1m\u001b[1;3m<final_answer>北京今天的天气是晴朗，温度为32摄氏度，无风。\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': '北京今天的天气如何?', 'output': '北京今天的天气是晴朗，温度为32摄氏度，无风。'}"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent_executor.invoke({\"input\": \"北京今天的天气如何?\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## agentType更新版\n",
    "![agtype.png](agtype.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一个比较复杂的agent\n",
    "- 工具\n",
    "- 检索增强RAG\n",
    "- 记忆\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义工具\n",
    "import os\n",
    "\n",
    "os.environ[\"SERPAPI_API_KEY\"] = \"f265b8d9834ed7692cba6db6618e2a8a9b24ed6964c457296a2626026e8ed594\"\n",
    "# Import things that are needed generically\n",
    "from langchain.pydantic_v1 import BaseModel, Field\n",
    "from langchain.tools import BaseTool, StructuredTool, tool\n",
    "from langchain_community.utilities import SerpAPIWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "search\n",
      "search(query: str) -> str - 当需要查找实时信息的时候才会使用这个工具.\n",
      "{'query': {'title': 'Query', 'type': 'string'}}\n"
     ]
    }
   ],
   "source": [
    "@tool\n",
    "def search(query: str) -> str:\n",
    "    \"\"\"当需要查找实时信息的时候才会使用这个工具.\"\"\"\n",
    "    serp = SerpAPIWrapper()\n",
    "    return serp.run(query)\n",
    "\n",
    "print(search.name)\n",
    "print(search.description)\n",
    "print(search.args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"{'type': 'weather_result', 'temperature': '53', 'unit': 'Fahrenheit', 'precipitation': '0%', 'humidity': '53%', 'wind': '5 mph', 'location': '中国北京市', 'date': 'Saturday', 'weather': 'Partly cloudy'}\""
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search(\"北京今天的天气如何?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 1 documents\n"
     ]
    }
   ],
   "source": [
    "#RAG增强生成\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "from langchain_community.vectorstores import FAISS\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "\n",
    "loader = WebBaseLoader(\"https://docs.smith.langchain.com/user_guide\")\n",
    "docs = loader.load()\n",
    "print(f\"Loaded {len(docs)} documents\")\n",
    "documents = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=1000, chunk_overlap=200\n",
    ").split_documents(docs)\n",
    "vector = FAISS.from_documents(documents, OpenAIEmbeddings())\n",
    "retriever = vector.as_retriever()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='The ability to rapidly understand how the model is performing ‚Äî and debug where it is failing ‚Äî is incredibly important for this phase.Debugging‚ÄãWhen developing new LLM applications, we suggest having LangSmith tracing enabled by default.\\nOftentimes, it isn‚Äôt necessary to look at every single trace. However, when things go wrong (an unexpected end result, infinite agent loop, slower than expected execution, higher than expected token usage), it‚Äôs extremely helpful to debug by looking through the application traces. LangSmith gives clear visibility and debugging information at each step of an LLM sequence, making it much easier to identify and root-cause issues.', metadata={'source': 'https://docs.smith.langchain.com/user_guide', 'title': 'LangSmith User Guide | \\uf8ffü¶úÔ∏è\\uf8ffüõ†Ô∏è LangSmith', 'description': 'LangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.', 'language': 'en'})"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#搜索匹配文档块\n",
    "retriever.get_relevant_documents(\"如何debug?\")[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tool(name='langsmith_search', description='搜索有关 LangSmith 的信息。关于LangSmith的任何问题，你一定要使用这个工具！', args_schema=<class 'langchain.tools.retriever.RetrieverInput'>, func=functools.partial(<function _get_relevant_documents at 0x14e139da0>, retriever=VectorStoreRetriever(tags=['FAISS', 'OpenAIEmbeddings'], vectorstore=<langchain_community.vectorstores.faiss.FAISS object at 0x108502cd0>), document_prompt=PromptTemplate(input_variables=['page_content'], template='{page_content}'), document_separator='\\n\\n'), coroutine=functools.partial(<function _aget_relevant_documents at 0x14e13a0c0>, retriever=VectorStoreRetriever(tags=['FAISS', 'OpenAIEmbeddings'], vectorstore=<langchain_community.vectorstores.faiss.FAISS object at 0x108502cd0>), document_prompt=PromptTemplate(input_variables=['page_content'], template='{page_content}'), document_separator='\\n\\n'))"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#把检索器加入到工具中\n",
    "from langchain.tools.retriever import create_retriever_tool\n",
    "retriever_tool = create_retriever_tool(\n",
    "    retriever,\n",
    "    \"langsmith_search\",\n",
    "    \"搜索有关 LangSmith 的信息。关于LangSmith的任何问题，你一定要使用这个工具！\",\n",
    ")\n",
    "retriever_tool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[StructuredTool(name='search', description='search(query: str) -> str - 当需要查找实时信息的时候才会使用这个工具.', args_schema=<class 'pydantic.v1.main.searchSchemaSchema'>, func=<function search at 0x14e9adbc0>),\n",
       " Tool(name='langsmith_search', description='搜索有关 LangSmith 的信息。关于LangSmith的任何问题，你一定要使用这个工具！', args_schema=<class 'langchain.tools.retriever.RetrieverInput'>, func=functools.partial(<function _get_relevant_documents at 0x14e139da0>, retriever=VectorStoreRetriever(tags=['FAISS', 'OpenAIEmbeddings'], vectorstore=<langchain_community.vectorstores.faiss.FAISS object at 0x108502cd0>), document_prompt=PromptTemplate(input_variables=['page_content'], template='{page_content}'), document_separator='\\n\\n'), coroutine=functools.partial(<function _aget_relevant_documents at 0x14e13a0c0>, retriever=VectorStoreRetriever(tags=['FAISS', 'OpenAIEmbeddings'], vectorstore=<langchain_community.vectorstores.faiss.FAISS object at 0x108502cd0>), document_prompt=PromptTemplate(input_variables=['page_content'], template='{page_content}'), document_separator='\\n\\n'))]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#可用工具集\n",
    "tools = [search, retriever_tool]\n",
    "tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义模型\n",
    "from langchain_openai import ChatOpenAI\n",
    "llm = ChatOpenAI(model=\"gpt-4-1106-preview\", temperature=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template='You are a helpful assistant')),\n",
       " MessagesPlaceholder(variable_name='chat_history', optional=True),\n",
       " HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['input'], template='{input}')),\n",
       " MessagesPlaceholder(variable_name='agent_scratchpad')]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#从hub中获取模版\n",
    "from langchain import hub\n",
    "\n",
    "# 一个最简单的模版,带记忆\n",
    "prompt = hub.pull(\"hwchase17/openai-functions-agent\")\n",
    "prompt.messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#创建agent\n",
    "from langchain.agents import create_openai_functions_agent\n",
    "agent = create_openai_functions_agent(llm, tools, prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "#创建agent执行器AgentExecutor\n",
    "from langchain.agents import AgentExecutor\n",
    "agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3mHello! How can I assist you today?\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': 'hi!', 'output': 'Hello! How can I assist you today?'}"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#执行\n",
    "agent_executor.invoke({\"input\": \"hi!\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m\n",
      "Invoking: `langsmith_search` with `{'query': '\\nlangsmith project testing'}`\n",
      "\n",
      "\n",
      "\u001b[0m\u001b[33;1m\u001b[1;3mWe provide native rendering of chat messages, functions, and retrieve documents.Initial Test Set‚ÄãWhile many developers still ship an initial version of their application based on ‚Äúvibe checks‚Äù, we‚Äôve seen an increasing number of engineering teams start to adopt a more test driven approach. LangSmith allows developers to create datasets, which are collections of inputs and reference outputs, and use these to run tests on their LLM applications.\n",
      "These test cases can be uploaded in bulk, created on the fly, or exported from application traces. LangSmith also makes it easy to run custom evaluations (both LLM and heuristic based) to score test results.Comparison View‚ÄãWhen prototyping different versions of your applications and making changes, it‚Äôs important to see whether or not you‚Äôve regressed with respect to your initial test cases.\n",
      "Oftentimes, changes in the prompt, retrieval strategy, or model choice can have huge implications in responses produced by your application.\n",
      "\n",
      "Skip to main contentü¶úÔ∏èüõ†Ô∏è LangSmith DocsLangChain Python DocsLangChain JS/TS DocsLangSmith API DocsSearchGo to AppLangSmithUser GuideSetupPricing (Coming Soon)Self-HostingTracingEvaluationMonitoringPrompt HubUser GuideOn this pageLangSmith User GuideLangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.Prototyping‚ÄãPrototyping LLM applications often involves quick experimentation between prompts, model types, retrieval strategy and other parameters.\n",
      "\n",
      "cost, and feedback scores. This ensures that it's delivering desirable results at scale.Monitoring and A/B Testing‚ÄãLangSmith provides monitoring charts that allow you to track key metrics over time. You can expand to view metrics for a given period and drill down into a specific data point to get a trace table for that time period ‚Äî this is especially handy for debugging production issues.LangSmith also allows for tag and metadata grouping, which allows users to mark different versions of their applications with different identifiers and view how they are performing side-by-side within each chart. This is helpful for A/B testing changes in prompt, model, or retrieval strategy.PreviousLangSmithNextSetupPrototypingBeta TestingProductionCommunityDiscordTwitterGitHubDocs CodeLangSmith SDKPythonJS/TSMoreHomepageBlogCopyright ¬© 2024 LangChain, Inc.\n",
      "\n",
      "LangSmith User Guide | ü¶úÔ∏èüõ†Ô∏è LangSmith\u001b[0m\u001b[32;1m\u001b[1;3mLangSmith can assist with project testing in several ways:\n",
      "\n",
      "1. **Creating Test Datasets**: Developers can create datasets that consist of collections of inputs and reference outputs. These datasets can be used to run tests on their LLM (Large Language Model) applications to ensure that the application behaves as expected.\n",
      "\n",
      "2. **Bulk Upload of Test Cases**: Test cases can be uploaded in bulk, which allows for efficient testing of multiple scenarios at once.\n",
      "\n",
      "3. **On-the-Fly Test Case Creation**: Developers can create test cases on the fly, which is useful for quickly testing new features or changes in the application.\n",
      "\n",
      "4. **Exporting from Application Traces**: Test cases can also be exported from application traces, which helps in creating tests based on real-world usage patterns.\n",
      "\n",
      "5. **Custom Evaluations**: LangSmith enables running custom evaluations, both LLM and heuristic-based, to score test results. This helps in assessing the performance of the application under various conditions.\n",
      "\n",
      "6. **Comparison View**: When making changes to the application, LangSmith provides a comparison view to see if there has been any regression with respect to the initial test cases. This is important when changes are made to prompts, retrieval strategies, or model choices.\n",
      "\n",
      "7. **Monitoring and A/B Testing**: LangSmith offers monitoring charts to track key metrics over time and allows for tag and metadata grouping. This feature is useful for marking different versions of applications and comparing their performance side-by-side, which is essential for A/B testing.\n",
      "\n",
      "Overall, LangSmith supports a test-driven approach to LLM application development, providing tools for creating, managing, and evaluating test cases, as well as monitoring and comparing application performance over time.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': 'langsmith如何帮助做项目测试?',\n",
       " 'output': 'LangSmith can assist with project testing in several ways:\\n\\n1. **Creating Test Datasets**: Developers can create datasets that consist of collections of inputs and reference outputs. These datasets can be used to run tests on their LLM (Large Language Model) applications to ensure that the application behaves as expected.\\n\\n2. **Bulk Upload of Test Cases**: Test cases can be uploaded in bulk, which allows for efficient testing of multiple scenarios at once.\\n\\n3. **On-the-Fly Test Case Creation**: Developers can create test cases on the fly, which is useful for quickly testing new features or changes in the application.\\n\\n4. **Exporting from Application Traces**: Test cases can also be exported from application traces, which helps in creating tests based on real-world usage patterns.\\n\\n5. **Custom Evaluations**: LangSmith enables running custom evaluations, both LLM and heuristic-based, to score test results. This helps in assessing the performance of the application under various conditions.\\n\\n6. **Comparison View**: When making changes to the application, LangSmith provides a comparison view to see if there has been any regression with respect to the initial test cases. This is important when changes are made to prompts, retrieval strategies, or model choices.\\n\\n7. **Monitoring and A/B Testing**: LangSmith offers monitoring charts to track key metrics over time and allows for tag and metadata grouping. This feature is useful for marking different versions of applications and comparing their performance side-by-side, which is essential for A/B testing.\\n\\nOverall, LangSmith supports a test-driven approach to LLM application development, providing tools for creating, managing, and evaluating test cases, as well as monitoring and comparing application performance over time.'}"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent_executor.invoke({\"input\": \"langsmith如何帮助做项目测试?\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m\n",
      "Invoking: `search` with `{'query': '\\nBeijing weather today'}`\n",
      "\n",
      "\n",
      "\u001b[0m\u001b[36;1m\u001b[1;3m{'type': 'weather_result', 'temperature': '53', 'unit': 'Fahrenheit', 'precipitation': '0%', 'humidity': '53%', 'wind': '5 mph', 'location': 'Beijing, China', 'date': 'Saturday', 'weather': 'Partly cloudy'}\u001b[0m\u001b[32;1m\u001b[1;3m今天北京的天气是部分多云，气温为53华氏度（约11.7摄氏度），降水概率为0%，湿度为53%，风速为每小时5英里。\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': '北京今天的天气如何?',\n",
       " 'output': '今天北京的天气是部分多云，气温为53华氏度（约11.7摄氏度），降水概率为0%，湿度为53%，风速为每小时5英里。'}"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#实时信息提问\n",
    "agent_executor.invoke({\"input\": \"北京今天的天气如何?\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m到目前为止，我们还没有开始任何讨论。这是我们的第一次互动。如果您有任何问题或需要帮助，请随时告诉我，我会尽力协助您！\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': '截止目前我们都聊了什么?',\n",
       " 'output': '到目前为止，我们还没有开始任何讨论。这是我们的第一次互动。如果您有任何问题或需要帮助，请随时告诉我，我会尽力协助您！'}"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#前面的交互都不带记忆\n",
    "agent_executor.invoke({\"input\": \"截止目前我们都聊了什么?\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m你好，Bob！很高兴认识你。有什么我可以帮助你的吗？\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': 'hi! 我叫bob',\n",
       " 'chat_history': [],\n",
       " 'output': '你好，Bob！很高兴认识你。有什么我可以帮助你的吗？'}"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#交互时添加记忆\n",
    "agent_executor.invoke({\"input\": \"hi! 我叫bob\", \"chat_history\": []})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m你的名字是Bob。有什么我可以帮你的吗？\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'chat_history': [HumanMessage(content='hi! 我叫bob'),\n",
       "  AIMessage(content='你好，Bob！很高兴认识你。有什么我可以帮助你的吗？')],\n",
       " 'input': '我叫什么名字?',\n",
       " 'output': '你的名字是Bob。有什么我可以帮你的吗？'}"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#手动构造记忆数据\n",
    "from langchain_core.messages import AIMessage, HumanMessage\n",
    "agent_executor.invoke(\n",
    "    {\n",
    "        \"chat_history\": [\n",
    "            HumanMessage(content=\"hi! 我叫bob\"),\n",
    "            AIMessage(content=\"你好，Bob！很高兴认识你。有什么我可以帮助你的吗？\"),\n",
    "        ],\n",
    "        \"input\": \"我叫什么名字?\",\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "#使用RunnableWithMessageHistory自动构造记忆数据\n",
    "from langchain_community.chat_message_histories import ChatMessageHistory\n",
    "from langchain_core.runnables.history import RunnableWithMessageHistory\n",
    "message_history = ChatMessageHistory()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent_with_chat_history = RunnableWithMessageHistory(\n",
    "    agent_executor,\n",
    "    #注意此处session_id没有用到，因为我们没用类似redis的存储,只是用了一个变量\n",
    "    lambda session_id: message_history,\n",
    "    input_messages_key=\"input\",\n",
    "    history_messages_key=\"chat_history\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m你好，Tomie！很高兴见到你。有什么我可以帮助你的吗？\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': 'hi! 我是tomie',\n",
       " 'chat_history': [],\n",
       " 'output': '你好，Tomie！很高兴见到你。有什么我可以帮助你的吗？'}"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#调用方式\n",
    "agent_with_chat_history.invoke(\n",
    "    {\"input\": \"hi! 我是tomie\"},\n",
    "    #注意此处session_id没有用到，因为我们没用类似redis的存储,只是用了一个变量\n",
    "    config={\"configurable\": {\"session_id\": \"foo_001\"}},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m你刚才告诉我你的名字是Tomie。有什么我可以帮你的吗？\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': '我叫什么名字?',\n",
       " 'chat_history': [HumanMessage(content='hi! 我是tomie'),\n",
       "  AIMessage(content='你好，Tomie！很高兴见到你。有什么我可以帮助你的吗？')],\n",
       " 'output': '你刚才告诉我你的名字是Tomie。有什么我可以帮你的吗？'}"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#调用方式\n",
    "agent_with_chat_history.invoke(\n",
    "    {\"input\": \"我叫什么名字?\"},\n",
    "    #注意此处session_id没有用到，因为我们没用类似redis的存储,只是用了一个变量\n",
    "    config={\"configurable\": {\"session_id\": \"foo_001\"}},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m\n",
      "Invoking: `langsmith_search` with `{'query': 'LangSmith \\n'}`\n",
      "\n",
      "\n",
      "\u001b[0m\u001b[33;1m\u001b[1;3mLangSmith User Guide | ü¶úÔ∏èüõ†Ô∏è LangSmith\n",
      "\n",
      "Skip to main contentü¶úÔ∏èüõ†Ô∏è LangSmith DocsLangChain Python DocsLangChain JS/TS DocsLangSmith API DocsSearchGo to AppLangSmithUser GuideSetupPricing (Coming Soon)Self-HostingTracingEvaluationMonitoringPrompt HubUser GuideOn this pageLangSmith User GuideLangSmith is a platform for LLM application development, monitoring, and testing. In this guide, we‚Äôll highlight the breadth of workflows LangSmith supports and how they fit into each stage of the application development lifecycle. We hope this will inform users how to best utilize this powerful platform or give them something to consider if they‚Äôre just starting their journey.Prototyping‚ÄãPrototyping LLM applications often involves quick experimentation between prompts, model types, retrieval strategy and other parameters.\n",
      "\n",
      "We provide native rendering of chat messages, functions, and retrieve documents.Initial Test Set‚ÄãWhile many developers still ship an initial version of their application based on ‚Äúvibe checks‚Äù, we‚Äôve seen an increasing number of engineering teams start to adopt a more test driven approach. LangSmith allows developers to create datasets, which are collections of inputs and reference outputs, and use these to run tests on their LLM applications.\n",
      "These test cases can be uploaded in bulk, created on the fly, or exported from application traces. LangSmith also makes it easy to run custom evaluations (both LLM and heuristic based) to score test results.Comparison View‚ÄãWhen prototyping different versions of your applications and making changes, it‚Äôs important to see whether or not you‚Äôve regressed with respect to your initial test cases.\n",
      "Oftentimes, changes in the prompt, retrieval strategy, or model choice can have huge implications in responses produced by your application.\n",
      "\n",
      "The ability to rapidly understand how the model is performing ‚Äî and debug where it is failing ‚Äî is incredibly important for this phase.Debugging‚ÄãWhen developing new LLM applications, we suggest having LangSmith tracing enabled by default.\n",
      "Oftentimes, it isn‚Äôt necessary to look at every single trace. However, when things go wrong (an unexpected end result, infinite agent loop, slower than expected execution, higher than expected token usage), it‚Äôs extremely helpful to debug by looking through the application traces. LangSmith gives clear visibility and debugging information at each step of an LLM sequence, making it much easier to identify and root-cause issues.\u001b[0m\u001b[32;1m\u001b[1;3mLangSmith是一个用于LLM（大型语言模型）应用开发、监控和测试的平台。以下是如何使用LangSmith的一些基本指南：\n",
      "\n",
      "1. **原型设计**：在LLM应用的原型设计阶段，通常涉及到快速在不同的提示（prompts）、模型类型、检索策略和其他参数之间进行实验。LangSmith提供了聊天消息、函数和检索文档的原生渲染。\n",
      "\n",
      "2. **初始测试集**：开发者可以创建数据集，这些数据集是输入和参考输出的集合，并使用这些数据集在LLM应用上运行测试。测试用例可以批量上传、即时创建或从应用跟踪中导出。LangSmith还可以轻松运行自定义评估（基于LLM和启发式的）来评分测试结果。\n",
      "\n",
      "3. **比较视图**：在原型设计不同版本的应用并进行更改时，重要的是要看到你是否在初始测试用例方面有所退步。提示、检索策略或模型选择的变化可能对应用程序产生的响应有很大影响。快速理解模型的性能并调试失败的地方对于这个阶段非常重要。\n",
      "\n",
      "4. **调试**：在开发新的LLM应用时，建议默认启用LangSmith跟踪。通常不需要查看每一个跟踪，但当出现问题（意外的结果、无限代理循环、执行速度慢于预期、令牌使用量高于预期）时，通过查看应用跟踪来调试非常有帮助。LangSmith在LLM序列的每一步提供清晰的可见性和调试信息，使得识别和找出问题根源变得更加容易。\n",
      "\n",
      "这些是LangSmith的一些基本使用指南。如果你有更具体的问题或需要进一步的帮助，请告诉我！\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': 'LangSmith如何使用?',\n",
       " 'chat_history': [HumanMessage(content='hi! 我是tomie'),\n",
       "  AIMessage(content='你好，Tomie！很高兴见到你。有什么我可以帮助你的吗？'),\n",
       "  HumanMessage(content='我叫什么名字?'),\n",
       "  AIMessage(content='你刚才告诉我你的名字是Tomie。有什么我可以帮你的吗？')],\n",
       " 'output': 'LangSmith是一个用于LLM（大型语言模型）应用开发、监控和测试的平台。以下是如何使用LangSmith的一些基本指南：\\n\\n1. **原型设计**：在LLM应用的原型设计阶段，通常涉及到快速在不同的提示（prompts）、模型类型、检索策略和其他参数之间进行实验。LangSmith提供了聊天消息、函数和检索文档的原生渲染。\\n\\n2. **初始测试集**：开发者可以创建数据集，这些数据集是输入和参考输出的集合，并使用这些数据集在LLM应用上运行测试。测试用例可以批量上传、即时创建或从应用跟踪中导出。LangSmith还可以轻松运行自定义评估（基于LLM和启发式的）来评分测试结果。\\n\\n3. **比较视图**：在原型设计不同版本的应用并进行更改时，重要的是要看到你是否在初始测试用例方面有所退步。提示、检索策略或模型选择的变化可能对应用程序产生的响应有很大影响。快速理解模型的性能并调试失败的地方对于这个阶段非常重要。\\n\\n4. **调试**：在开发新的LLM应用时，建议默认启用LangSmith跟踪。通常不需要查看每一个跟踪，但当出现问题（意外的结果、无限代理循环、执行速度慢于预期、令牌使用量高于预期）时，通过查看应用跟踪来调试非常有帮助。LangSmith在LLM序列的每一步提供清晰的可见性和调试信息，使得识别和找出问题根源变得更加容易。\\n\\n这些是LangSmith的一些基本使用指南。如果你有更具体的问题或需要进一步的帮助，请告诉我！'}"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#调用方式\n",
    "agent_with_chat_history.invoke(\n",
    "    {\"input\": \"LangSmith如何使用?\"},\n",
    "    #注意此处session_id没有用到，因为我们没用类似redis的存储,只是用了一个变量\n",
    "    config={\"configurable\": {\"session_id\": \"foo_001\"}},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m到目前为止，我们的对话内容如下：\n",
      "\n",
      "1. 你向我打招呼，并告诉我你的名字是Tomie。\n",
      "2. 我问你是否需要帮助。\n",
      "3. 你询问了你的名字，我确认了你之前告诉我的名字是Tomie。\n",
      "4. 你问了关于LangSmith如何使用的问题，我给你提供了关于LangSmith使用的一些基本指南。\n",
      "\n",
      "如果你有其他问题或需要进一步的帮助，请随时告诉我！\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': '截止目前我们都聊了什么?',\n",
       " 'chat_history': [HumanMessage(content='hi! 我是tomie'),\n",
       "  AIMessage(content='你好，Tomie！很高兴见到你。有什么我可以帮助你的吗？'),\n",
       "  HumanMessage(content='我叫什么名字?'),\n",
       "  AIMessage(content='你刚才告诉我你的名字是Tomie。有什么我可以帮你的吗？'),\n",
       "  HumanMessage(content='LangSmith如何使用?'),\n",
       "  AIMessage(content='LangSmith是一个用于LLM（大型语言模型）应用开发、监控和测试的平台。以下是如何使用LangSmith的一些基本指南：\\n\\n1. **原型设计**：在LLM应用的原型设计阶段，通常涉及到快速在不同的提示（prompts）、模型类型、检索策略和其他参数之间进行实验。LangSmith提供了聊天消息、函数和检索文档的原生渲染。\\n\\n2. **初始测试集**：开发者可以创建数据集，这些数据集是输入和参考输出的集合，并使用这些数据集在LLM应用上运行测试。测试用例可以批量上传、即时创建或从应用跟踪中导出。LangSmith还可以轻松运行自定义评估（基于LLM和启发式的）来评分测试结果。\\n\\n3. **比较视图**：在原型设计不同版本的应用并进行更改时，重要的是要看到你是否在初始测试用例方面有所退步。提示、检索策略或模型选择的变化可能对应用程序产生的响应有很大影响。快速理解模型的性能并调试失败的地方对于这个阶段非常重要。\\n\\n4. **调试**：在开发新的LLM应用时，建议默认启用LangSmith跟踪。通常不需要查看每一个跟踪，但当出现问题（意外的结果、无限代理循环、执行速度慢于预期、令牌使用量高于预期）时，通过查看应用跟踪来调试非常有帮助。LangSmith在LLM序列的每一步提供清晰的可见性和调试信息，使得识别和找出问题根源变得更加容易。\\n\\n这些是LangSmith的一些基本使用指南。如果你有更具体的问题或需要进一步的帮助，请告诉我！')],\n",
       " 'output': '到目前为止，我们的对话内容如下：\\n\\n1. 你向我打招呼，并告诉我你的名字是Tomie。\\n2. 我问你是否需要帮助。\\n3. 你询问了你的名字，我确认了你之前告诉我的名字是Tomie。\\n4. 你问了关于LangSmith如何使用的问题，我给你提供了关于LangSmith使用的一些基本指南。\\n\\n如果你有其他问题或需要进一步的帮助，请随时告诉我！'}"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#调用方式\n",
    "agent_with_chat_history.invoke(\n",
    "    {\"input\": \"截止目前我们都聊了什么?\"},\n",
    "    #注意此处session_id没有用到，因为我们没用类似redis的存储,只是用了一个变量\n",
    "    config={\"configurable\": {\"session_id\": \"foo_001\"}},\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "langchains",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
    